from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

# step1: load all the 100KQueryTerms with their term freq in collection to the main memory using a dict structure
# the size of this dict will be: 38871
# key: term 
# value: term_freq_in_collection

newTermPopUp = 0

queryTermBeingTouchCurrentlyDict = {}
queryTermBeingTouchCurrentlyList = []

numOfUniquePostingsBeingTouchedCurrently = 0

queryTermDictWithFreqInCollection = {}

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueryTermsWithTermFreqInCollection.txt"
inputFileHandler = open(inputFileName,"r")

dataLine = inputFileHandler.readline()

while dataLine:
    dataLineElements = dataLine.strip().split(" ")
    queryTerm = dataLineElements[0]
    queryTermFreqInCollection = int(dataLineElements[1])
    if queryTerm not in queryTermDictWithFreqInCollection:
        queryTermDictWithFreqInCollection[queryTerm] = queryTermFreqInCollection
    else:
        print "error, mark1"
    
    # print "weiIndexPostingCount:",weiIndexPostingCount
    dataLine = inputFileHandler.readline()

print "len(queryTermDictWithFreqInCollection):",len(queryTermDictWithFreqInCollection)
inputFileHandler.close()

# step2: load a certain fraction of queries in the 100K to see how many postings have been covered.
queryDict = {}
inputQueryFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueries"
inputQueryHandler = open(inputQueryFileName,"r")
for index,line in enumerate( inputQueryHandler.readlines() ):
    elements = line.strip().split(":")
    queryID = int(elements[0])
    
    data = elements[1]
    data = data.lower()
    
    for i in range(0,len(data)):
        # print "data[i]:",ord(data[i])
        if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
            # Just replace them with a space.
            data = data[:i] + " " + data[i+1:]
    
    queryContent = data

    if queryID not in queryDict:
        queryDict[queryID] = queryContent
        # Let's do some content analysis on this
        queryContentElements = queryContent.strip().split(" ")
        for term in queryContentElements:
            if term.strip() != "" and term.strip() in queryTermDictWithFreqInCollection:
                if term.strip() not in queryTermBeingTouchCurrentlyDict:
                    queryTermBeingTouchCurrentlyDict[ term.strip() ] = 1
                    numOfUniquePostingsBeingTouchedCurrently += queryTermDictWithFreqInCollection[term.strip()]
                    newTermPopUp += 1
                else:
                    queryTermBeingTouchCurrentlyDict[ term.strip() ] += 1
            else:
                if term.strip() != "":
                    print "the term:",term.strip(),"is NOT in the lexicon"
    
    if (index+1) == 2:
        print index+1,numOfUniquePostingsBeingTouchedCurrently,newTermPopUp
        newTermPopUp = 0
    
    if (index+1) % 10000 == 0:  
        print index+1,numOfUniquePostingsBeingTouchedCurrently,newTermPopUp
        newTermPopUp = 0

    '''
    print "queryTermBeingTouchCurrentlyDict:",queryTermBeingTouchCurrentlyDict
    tempCheckPostingCounter = 0
    for element in queryTermBeingTouchCurrentlyDict:
        tempCheckPostingCounter += queryTermDictWithFreqInCollection[element]
        print element," ",queryTermDictWithFreqInCollection[element]
    print "tempCheckPostingCounter:",tempCheckPostingCounter
    '''

for queryTerm in queryTermBeingTouchCurrentlyDict:
    queryTermBeingTouchCurrentlyList.append( (queryTerm,queryTermBeingTouchCurrentlyDict[queryTerm]) )

queryTermBeingTouchCurrentlyList.sort(cmp=None, key=itemgetter(1), reverse=True)

for tuple in queryTermBeingTouchCurrentlyList[0:10]:
    print tuple
    
print "len(queryTermBeingTouchCurrentlyDict):",len(queryTermBeingTouchCurrentlyDict)


     
# print "----->","len(queryDict):",len(queryDict)
inputQueryHandler.close()